from gevent import monkey; monkey.patch_all()
import time
from lxml import etree
import gevent
import requests

url = r'https://www.boxofficemojo.com/quarter/q2/2020/?grossesOption=totalGrosses'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
rsp = requests.get(url, headers=headers)
text = rsp.text
html = etree.HTML(text)
movie_relative_urls =  html.xpath(r'//td[@class="a-text-left mojo-field-type-release mojo-cell-wide"]/a/@href')
movie_urls = [r'https://www.boxofficemojo.com'+u for u in movie_relative_urls]
genres_dict = {}
task_list = []

def spider(url):
    rsp = requests.get(url, headers=headers)
    text = rsp.text
    html = etree.HTML(text)
    genre = html.xpath(r'//div/span[text()="Genres"]/following-sibling::span[1]/text()')[0]
    title = html.xpath(r'//div/h1/text()')[0]
    genres_dict[title] = genre
    
    
gevent_start = time.time()
for u in movie_urls:
    task = gevent.spawn(spider, u)
    task_list.append(task)
    
gevent.joinall(task_list)
gevent_end = time.time()
gevent_elapse = gevent_end - gevent_start
print('The gevent spider costs %s seconds' % gevent_elapse)
 